from urllib.parse import urlencode
from requests.exceptions import RequestException
from bs4 import BeautifulSoup
import re
import json
import requests

def get_page_index(offset,keyword):
    data =  {
        'offset': offset,
        'format' : 'json',
        'keyword' : keyword,
        'autoload' : 'true',
        'count' : '20',
        'cur_tab' : 3,
    }
    url = 'http://www.toutiao.com/search_content/?'+urlencode(data)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('请求索引页出错')
        return None

def parse_page_index(html):
    data = json.loads(html)
    if data and 'data' in data.keys():
        for item in data.get('data'):
            yield item.get('article_url')
def get_page_detail(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('请求详情页出错',url)
        return

def parse_page_detail(html,url):
    soup = BeautifulSoup(html)
    title = soup.select('title')[0].get_text()
    images_pattern = re.compile('var gallery = (.*?);', re.S)
    result = re.search(images_pattern,html)
    if result:
        data = json.loads(result.group(1))
        if data and 'sub_images' in data.keys():
            sub_images = data.get('sub_images')
            images = [item.get('url') for item in sub_images]
            return {
                'title': title,
                'url': url,
                'images': images
            }
def main():
    html =  get_page_index(0,'街拍')
    for url in parse_page_index(html):
        html = get_page_detail(url)
        if html:
            result = parse_page_detail(html,url)
            print(result.get('title'))
            str_need = ''
            for i in result.get('images'):
                str_need = str_need + '   ' + str(i)
            print(str_need)
            f = open('test.txt', 'w+')
            f.write(result.get('title'))
            f.write(str_need)
            f.close()


if __name__ == '__main__':
    main()
